// -*- mode: groovy -*-
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.
//
//This is a Jenkinsfile for nightly tests. The format and some functions have been picked up from the top-level Jenkinsfile

mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, lib/libtvm_runtime.so, lib/libtvmop.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a'
mx_cmake_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so'
mx_lib_cpp_example_mkl = 'lib/libmxnet.so, lib/libmxnet.a, lib/libtvm_runtime.so, lib/libtvmop.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, build/cpp-package/example/imagenet_inference, lib/libmkldnn.so.1'

node('utility') {
  // Loading the utilities requires a node context unfortunately
  checkout scm
  utils = load('ci/Jenkinsfile_utils.groovy')
}
utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu', linux_gpu: 'mxnetlinux-gpu', linux_gpu_p3: 'mxnetlinux-gpu-p3', windows_cpu: 'mxnetwindows-cpu', windows_gpu: 'mxnetwindows-gpu')

utils.main_wrapper(
core_logic: {
  stage('Build') {
    parallel 'GPU: CUDA10.1+cuDNN7': {
      node(NODE_LINUX_CPU) {
        ws('workspace/build-mkldnn-gpu') {
          utils.init_git()
          utils.docker_run('ubuntu_build_cuda', 'build_ubuntu_gpu_mkldnn', false)
          utils.pack_lib('gpu', mx_lib_cpp_example_mkl)
        }
      }
    }/*,
    'CPU: USE_INT64_TENSOR_SIZE': {
      node(NODE_LINUX_CPU) {
        ws('workspace/build-cpu-int64') {
          utils.init_git()
          utils.docker_run('ubuntu_nightly_cpu', 'build_ubuntu_cpu_large_tensor', false)
          utils.pack_lib('cpu_int64', mx_cmake_lib)
        }
      }
    },
    'GPU: USE_INT64_TENSOR_SIZE': {
      node(NODE_LINUX_GPU) {
        ws('workspace/build-gpu-int64') {
          utils.init_git()
          utils.docker_run('ubuntu_nightly_gpu', 'build_ubuntu_gpu_large_tensor', true)
          utils.pack_lib('gpu_int64', mx_cmake_lib)
        }
      }
    }*/
  }

  stage('NightlyTests'){
    parallel 'ImageClassification: GPU': {
      node(NODE_LINUX_GPU) {
        ws('workspace/nt-ImageClassificationTest') {
          utils.unpack_and_init('gpu', mx_lib)
          utils.docker_run('ubuntu_nightly_gpu', 'nightly_test_image_classification', true)
        }
      }
    },
    'ImageNet Inference: GPU': {
      node(NODE_LINUX_GPU) {
        ws('workspace/nt-ImageInferenceTest') {
          utils.unpack_and_init('gpu', mx_lib_cpp_example_mkl)
          utils.docker_run('ubuntu_nightly_gpu', 'nightly_test_imagenet_inference', true)
        }
      }
    },
    'KVStore_SingleNode: GPU': {
      node('mxnetlinux-gpu-p3-8xlarge') {
        ws('workspace/nt-KVStoreTest') {
          utils.unpack_and_init('gpu', mx_lib)
          utils.docker_run('ubuntu_nightly_gpu', 'nightly_test_KVStore_singleNode', true)
        }
      }
    },
    // https://github.com/apache/incubator-mxnet/issues/14980
    /*'Test Large Tensor Size: CPU': {
      node(NODE_LINUX_CPU) {
        ws('workspace/large_tensor-cpu') {
            utils.unpack_and_init('cpu_int64', mx_cmake_lib)
            utils.docker_run('ubuntu_nightly_cpu', 'nightly_test_large_tensor', false)
        }
      }
    },*/
    // https://github.com/apache/incubator-mxnet/issues/14981
    /*'Test Large Tensor Size: GPU': {
      node(NODE_LINUX_GPU) {
        ws('workspace/large_tensor-gpu') {
            utils.unpack_and_init('gpu_int64', mx_cmake_lib)
            utils.docker_run('ubuntu_nightly_gpu', 'nightly_test_large_tensor', true)
        }
      }
    },*/
    'StraightDope: Python2 Single-GPU': {
      node(NODE_LINUX_GPU_P3) {
        ws('workspace/straight_dope-single_gpu') {
          utils.unpack_and_init('gpu', mx_lib)
          utils.docker_run('ubuntu_nightly_gpu', 'nightly_straight_dope_python2_single_gpu_tests', true)
        }
      }
    },
    'StraightDope: Python2 Multi-GPU': {
      node(NODE_LINUX_GPU) {
        ws('workspace/straight_dope-multi_gpu') {
          utils.unpack_and_init('gpu', mx_lib)
          utils.docker_run('ubuntu_nightly_gpu', 'nightly_straight_dope_python2_multi_gpu_tests', true)
        }
      }
    },
    'StraightDope: Python3 Single-GPU': {
      node(NODE_LINUX_GPU_P3) {
        ws('workspace/straight_dope-single_gpu') {
          utils.unpack_and_init('gpu', mx_lib)
          utils.docker_run('ubuntu_nightly_gpu', 'nightly_straight_dope_python3_single_gpu_tests', true)
        }
      }
    },
    'StraightDope: Python3 Multi-GPU': {
      node(NODE_LINUX_GPU) {
        ws('workspace/straight_dope-multi_gpu') {
          utils.unpack_and_init('gpu', mx_lib)
          utils.docker_run('ubuntu_nightly_gpu', 'nightly_straight_dope_python3_multi_gpu_tests', true)
        }
      }
    },
    'Gluon estimator: GPU': {
      node(NODE_LINUX_GPU) {
        ws('workspace/estimator-test-gpu') {
          utils.unpack_and_init('gpu', mx_lib)
          utils.docker_run('ubuntu_nightly_gpu', 'nightly_estimator', true)
        }
      }
    }
  }
}
,
failure_handler: {
  if (currentBuild.result == "FAILURE") {
    emailext body: 'Nightly tests for MXNet branch ${BRANCH_NAME} failed. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[NIGHTLY TEST FAILED] build ${BUILD_NUMBER}', to: '${EMAIL}'
  }
}
)
